import numpy as np

from IPython import get_ipython

from IPython.display import display, HTML, Markdown

import ipywidgets as widgets

​

%matplotlib inline

%config InlineBackend.figure_format = 'retina'

​

import matplotlib.pyplot as plt

import seaborn as sns

sns.set()

import networkx as nx

​

import pandas as pd

pd.set_option('display.max_rows', 500)

pd.set_option('display.max_columns', 5000)

pd.set_option('display.max_colwidth', 5000)

​

# import plotly for visualization

​

# Standard plotly imports

import chart_studio.plotly as py

import plotly.graph_objs as go

from plotly.offline import iplot, init_notebook_mode

​

import plotly.express as px

​

# Using plotly + cufflinks in offline mode

import cufflinks as cf

cf.go_offline()

cf.set_config_file(offline=False, world_readable=True)

df = pd.read_excel('GDS_EY_Badges_Earned8528efa.xlsx')

df.head()

df.info()

#df['Badge earned'] = str(df['Badge earned'])

df['Badge earned'] = pd.to_datetime(df['Badge earned'])

df = df.sort_values(by = 'Badge earned')

df.groupby('Badge earned').count().iplot()

df.head()

process_list = df['Rank Name'].to_list()

#process_list.extend(df['Domain'].to_list())

process_list.extend(df['Sub Domain'].to_list())

​

process_list = (set(process_list))

proc_list = pd.DataFrame(process_list, columns = ['process_list'])

from sklearn.preprocessing import LabelEncoder

labelencoder_X_1 = LabelEncoder()

proc_list.loc[:, 'process_list_encoded'] = labelencoder_X_1.fit_transform(proc_list.loc[:, 'process_list'])

proc_list = proc_list.sort_values(by = 'process_list_encoded').reset_index(drop = True)

proc_list

proc_list['Type'] = 'Type1'

df_node = proc_list[['process_list','process_list_encoded', 'Type']]

df_node.shape

res = dict(zip(proc_list.process_list, proc_list.process_list_encoded))

df['Parent_process_encoded'] = df['Rank Name'].map(res)

df['New_process_encoded'] = df['Sub Domain'].map(res)

#df_node = df_node.drop_duplicates()

#unique_proc = pd.DataFrame({'process_count' : df.groupby([ 'Initiate a badge date','Rank Name','Domain','Sub Domain']).size()}).reset_index()

df_edge = df[['Parent_process_encoded','New_process_encoded','Initiate a badge date']]

#df_edge = unique_proc

df_edge.head()

df_edge['Initiate a badge date'] = df_edge['Initiate a badge date'].astype('str')

edge1 = df_edge

node1 = df_node

edge1.head()

​

G = nx.from_pandas_edgelist(edge1, 'Parent_process_encoded', 'New_process_encoded', ['Parent_process_encoded', 'New_process_encoded', 'Initiate a badge date'], create_using=nx.MultiDiGraph())

nx.set_node_attributes(G, node1.set_index('process_list_encoded')['process_list'].to_dict(), 'process_list')

nx.set_node_attributes(G, node1.set_index('process_list_encoded')['Type'].to_dict(), 'Type')

pos = nx.layout.spring_layout(G)

df.head()

g = nx.from_pandas_edgelist(df,source = 'Parent_process_encoded', target = 'New_process_encoded',edge_attr = ['Initiate a badge date'],create_using=nx.DiGraph())

print(nx.info(g))

pos=nx.spring_layout(g)

nx.draw(g,pos,node_color='#A0CBE2',edge_color='#00bb5e',width=1,edge_cmap=plt.cm.Blues,with_labels=True)

​

#No. of unique nodes

len(g.nodes)

import random

edges = dict()

for edge in g.edges:

    edges[(edge[0], edge[1])] = 1

print(nx.info(g))

import random

missing_edges = set([])

while (len(missing_edges)< 69):

    a=random.randint(1, 362)

    b=random.randint(1, 362)

    tmp = edges.get((a,b),-1)

    if tmp == -1 and a!=b:

        try:

            if nx.shortest_path_length(g,source=a,target=b) > 2: 

                missing_edges.add((a,b))

            else:

                continue  

        except:  

                missing_edges.add((a,b))              

    else:

        continue

len(missing_edges)

missing_edges

import pickle

pickle.dump(missing_edges,open('missing_edges_final.p','wb'))

import pickle

missing_edges = pickle.load(open('missing_edges_final.p','rb'))

df.head()

#reading total data df

df_pos = df[['Parent_process_encoded','New_process_encoded']]

df_neg = pd.DataFrame(list(missing_edges), columns=['Parent_process_encoded', 'New_process_encoded'])

#positive links i.e graph

df_pos.info()

#negative links synthesized links

df_neg.info()

#Trian test split 

#Spiltted data into 80-20 

#positive links and negative links seperatly because we need positive training data only for creating graph 

#and for feature generation

from sklearn.model_selection import train_test_split

X_train_pos, X_test_pos, y_train_pos, y_test_pos  = train_test_split(df_pos,np.ones(len(df_pos)),test_size=0.2, random_state=9)

X_train_neg, X_test_neg, y_train_neg, y_test_neg  = train_test_split(df_neg,np.zeros(len(df_neg)),test_size=0.2, random_state=9)

df_train_pos = pd.DataFrame(X_train_pos)

#removing header and saving

df_train_pos.to_csv('train_data.csv',header=False,index=False)

#Graph from Traing data only 

g=nx.read_edgelist('train_data.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)

print(nx.info(g))

#Graph of total data without splitting 

G=nx.from_pandas_edgelist(df,source = 'Parent_process_encoded', target = 'New_process_encoded',create_using=nx.DiGraph())

print(nx.info(G))

###total nodes in train positive data

train_nodes_pos = set(g.nodes())

###total nodes in total data

total_nodes_pos = set(G.nodes())

###test nodes in pos data

test_nodes_pos = set(X_test_pos.values.flatten())

print('no of process common in train and test -- ',len(train_nodes_pos.intersection(test_nodes_pos)))

print('no of process present in train but not present in test -- ',len(train_nodes_pos - test_nodes_pos))

print('no of process present in test but not present in train -- ',len(test_nodes_pos - train_nodes_pos))

print(' % of processes not there in Train but exist in Test in total Test data are {} %'.\

                  format(len(test_nodes_pos - train_nodes_pos)/len(test_nodes_pos)*100))

#final train and test data sets

df_final_train = X_train_pos.append(X_train_neg,ignore_index=True)

y_final_train = np.concatenate((y_train_pos,y_train_neg))

df_final_test = X_test_pos.append(X_test_neg,ignore_index=True)

y_final_test = np.concatenate((y_test_pos,y_test_neg))

df_final_train.info()

df_final_test.info()

Adj = nx.adjacency_matrix(g,nodelist=sorted(g.nodes()))

Adj = Adj.asfptype()

Adj

from scipy.sparse.linalg import svds, eigs

U, s, V = svds(Adj,k = 3)

print('U Shape',U.shape)

print('V Shape',V.shape)

print('s Shape',s.shape)

del Adj

del s

#for followees

def jaccard_for_followees(a,b):

    try:

        if len(set(g.successors(a))) == 0  | len(set(g.successors(b))) == 0:

            return 0

        sim = (len(set(g.successors(a)).intersection(set(g.successors(b)))))/\

                                    (len(set(g.successors(a)).union(set(g.successors(b)))))

    except:

        return 0

    return sim

#for followers

def jaccard_for_followers(a,b):

    try:

        if len(set(g.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:

            return 0

        sim = (len(set(g.predecessors(a)).intersection(set(g.predecessors(b)))))/\

                                 (len(set(g.predecessors(a)).union(set(g.predecessors(b)))))

        return sim

    except:

        return 0

#for followees

def cosine_for_followees(a,b):

    try:

        if len(set(g.successors(a))) == 0  | len(set(g.successors(b))) == 0:

            return 0

        sim = (len(set(g.successors(a)).intersection(set(g.successors(b)))))/\

                                    (math.sqrt(len(set(g.successors(a)))*len((set(g.successors(b))))))

        return sim

    except:

        return 0

def cosine_for_followers(a,b):

    try:

        if len(set(g.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:

            return 0

        sim = (len(set(g.predecessors(a)).intersection(set(g.predecessors(b)))))/\

                                     (math.sqrt(len(set(g.predecessors(a))))*(len(set(g.predecessors(b)))))

        return sim

    except:

        return 0

pr = nx.pagerank(g, alpha=0.85)

pickle.dump(pr,open('page_rank.p','wb'))

pr = pickle.load(open('page_rank.p','rb'))

print('min',pr[min(pr, key=pr.get)])

print('max',pr[max(pr, key=pr.get)])

print('mean',float(sum(pr.values())) / len(pr))

#for imputing to nodes which are not there in Train data

mean_pr = float(sum(pr.values())) / len(pr)

#if has direct edge then deleting that edge and calculating shortest path

def compute_shortest_path_length(a,b):

    p=-1

    try:

        if g.has_edge(a,b):

            g.remove_edge(a,b)

            p= nx.shortest_path_length(g,source=a,target=b)

            g.add_edge(a,b)

        else:

            p= nx.shortest_path_length(g,source=a,target=b)

        return p

    except:

        return -1

#getting weekly connected edges from graph 

wcc=list(nx.weakly_connected_components(g))

def belongs_to_same_wcc(a,b):

    index = []

    if g.has_edge(b,a):

        return 1

    if g.has_edge(a,b):

            for i in wcc:

                if a in i:

                    index= i

                    break

            if (b in index):

                g.remove_edge(a,b)

                if compute_shortest_path_length(a,b)==-1:

                    g.add_edge(a,b)

                    return 0

                else:

                    g.add_edge(a,b)

                    return 1

            else:

                return 0

    else:

            for i in wcc:

                if a in i:

                    index= i

                    break

            if(b in index):

                return 1

            else:

                return 0

#adar index

def calc_adar_in(a,b):

    sum=0

    try:

        n=list(set(g.successors(a)).intersection(set(g.successors(b))))

        if len(n)!=0:

            for i in n:

                sum=sum+(1/np.log10(len(list(g.predecessors(i)))))

            return sum

        else:

            return 0

    except:

        return 0

def follows_back(a,b):

    if g.has_edge(b,a):

        return 1

    else:

        return 0

katz = nx.katz.katz_centrality(g,alpha=0.005,beta=1)

print('min',katz[min(katz, key=katz.get)])

print('max',katz[max(katz, key=katz.get)])

print('mean',float(sum(katz.values())) / len(katz))

mean_katz = float(sum(katz.values())) / len(katz)

hits = nx.hits(g, max_iter=100, tol=1e-08, nstart=None, normalized=True)

#hits = pickle.load(open('hits.p','rb'))

print('min',hits[0][min(hits[0], key=hits[0].get)])

print('max',hits[0][max(hits[0], key=hits[0].get)])

print('mean',float(sum(hits[0].values())) / len(hits[0]))

pickle.dump(hits,open('hits.p','wb'))

#mapping jaccrd followers to train data

df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:

                                        jaccard_for_followers(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

#mapping jaccrd followers to test data

df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:

                                        jaccard_for_followers(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

#mapping jaccrd followees to train and test data

df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:

                                        jaccard_for_followees(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:

                                        jaccard_for_followees(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

#calculating no of followers followees for source and destination

#calculating intersection of followers and followees for source and destination

num_followers_s=[]

num_followees_s=[]

num_followers_d=[]

num_followees_d=[]

inter_followers=[]

inter_followees=[]

for i,row in df_final_train.iterrows():

    try:

        s1=set(g.predecessors(row['Parent_process_encoded']))

        s2=set(g.successors(row['Parent_process_encoded']))

    except:

        s1 = set()

        s2 = set()

    try:

        d1=set(g.predecessors(row['New_process_encoded']))

        d2=set(g.successors(row['New_process_encoded']))

    except:

        d1 = set()

        d2 = set()

    num_followers_s.append(len(s1))

    num_followees_s.append(len(s2))

    num_followers_d.append(len(d1))

    num_followees_d.append(len(d2))

    inter_followers.append(len(s1.intersection(d1)))

    inter_followees.append(len(s2.intersection(d2)))

df_final_train['num_followers_s']=num_followers_s

df_final_train['num_followees_s']=num_followees_s

df_final_train['num_followers_d']=num_followers_d

df_final_train['num_followees_d']=num_followees_d

df_final_train['inter_followers']=inter_followers

df_final_train['inter_followees']=inter_followees

#For test data

#calculating no of followers followees for source and destination

#calculating intersection of followers and followees for source and destination

num_followers_s=[]

num_followees_s=[]

num_followers_d=[]

num_followees_d=[]

inter_followers=[]

inter_followees=[]

for i,row in df_final_test.iterrows():

    try:

        s1=set(g.predecessors(row['Parent_process_encoded']))

        s2=set(g.successors(row['Parent_process_encoded']))

    except:

        s1 = set()

        s2 = set()

    try:

        d1=set(g.predecessors(row['New_process_encoded']))

        d2=set(g.successors(row['New_process_encoded']))

    except:

        d1 = set()

        d2 = set()

    num_followers_s.append(len(s1))

    num_followees_s.append(len(s2))

    num_followers_d.append(len(d1))

    num_followees_d.append(len(d2))

    inter_followers.append(len(s1.intersection(d1)))

    inter_followees.append(len(s2.intersection(d2)))

#assigning

df_final_test['num_followers_s']=num_followers_s

df_final_test['num_followees_s']=num_followees_s

df_final_test['num_followers_d']=num_followers_d

df_final_test['num_followees_d']=num_followees_d

df_final_test['inter_followers']=inter_followers

df_final_test['inter_followees']=inter_followees

#saving to disk 

df_final_train.to_csv('df_final_train_some.csv',index=False)

df_final_test.to_csv('df_final_test_some.csv',index=False)

#head of df 

df_final_train.head()

df_final_test.head()

#mapping adar index on train

df_final_train['adar_index'] = df_final_train.apply(lambda row:

                                        calc_adar_in(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

​

#mapping adar index on test

df_final_test['adar_index'] = df_final_test.apply(lambda row:

                                        calc_adar_in(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

#mapping followback or not on train

df_final_train['follows_back'] = df_final_train.apply(lambda row:

                                        follows_back(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

​

#mapping followback or not on test

df_final_test['follows_back'] = df_final_test.apply(lambda row:

                                        follows_back(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

##mapping same component of wcc or not on train

df_final_train['same_comp'] = df_final_train.apply(lambda row:

                                        belongs_to_same_wcc(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

​

##mapping same component of wcc or not on train

df_final_test['same_comp'] = df_final_test.apply(lambda row:

                                        belongs_to_same_wcc(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

#saving to disk beacuse above operation takes much time so at every check point saving to disk 

df_final_train.to_csv('df_final_train_some1.csv',index=False)

df_final_test.to_csv('df_final_test_some1.csv',index=False)

df_final_train = pd.read_csv('df_final_train_some1.csv')

df_final_test = pd.read_csv('df_final_test_some1.csv')

#mapping shortest path on train 

df_final_train['shortest_path'] = df_final_train.apply(lambda row:

                                        compute_shortest_path_length(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

​

#mapping shortest path on test

df_final_test['shortest_path'] = df_final_test.apply(lambda row:

                                        compute_shortest_path_length(row['Parent_process_encoded'],row['New_process_encoded']),axis=1)

​

df_final_train.to_csv('df_final_train_some2.csv',index=False)

df_final_test.to_csv('df_final_test_some2.csv',index=False)

#page rank for source and destination in Train

#if anything not there in train graph then adding mean page rank 

df_final_train['page_rank_s'] = df_final_train.Parent_process_encoded.apply(lambda x:pr.get(x,mean_pr))

df_final_train['page_rank_d'] = df_final_train.New_process_encoded.apply(lambda x:pr.get(x,mean_pr))